# 基于聚类的无监督算法
from itertools import islice
import urllib.parse as urlPaese
import pymysql
import re


# 样本间距离
# url编码过滤
# senstive = "!@#$%^&*()_+-=~{}[]\|:\";\'<>?,./"
senstive = set("!@#$%^&*()_+-=~{}[]\|:\";\'<>?,./")
alpha = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
num = set("1234567890")
def ifContainNum(s):
    return s.intersection(num) != set()
def ifContainAlpha(s):
    return s.intersection(alpha) != set()
def ifContainAlphaAndNum(s):
    return ifContainAlpha(s) and ifContainNum(s)
def ifContainSenstive(s):
    return s.intersection(senstive)!=set()
def killAnUnseen(s):
    try:
        f = open('kill.html', 'w');
        f.write(s);
        f.close()
    except UnicodeEncodeError as err:
        info = str(err);
        st = re.search('\\\\U[a-f0-9]{8}|\\\\u[a-f0-9]{4}|\\\\x[a-f0-9]{2}', info).group()[2:];
        x = int(st, 16);
        return (s.replace(chr(x), ""), True)
    return s, False


def deUrlCode(dic):
    for x in list(dic.keys()):
        rst = []
        for y in dic[x]:
            afterUnquto = urlPaese.unquote(killUnseen(urlPaese.unquote(y)))
            afterUnquto = afterUnquto.replace('\r', '')
            afterUnquto = afterUnquto.replace('\n', '')
            rst.append(afterUnquto)
        dic[x] = rst


def killUnseen(s):
    while(True):
        s, res = killAnUnseen(s)
        if res == False:
            return s


# 2-grams的处理方法
def n_grams(a):
    z = (islice(a, i, None) for i in range(2))
    return list(zip(*z))

# 字符组合差异
def groupdiffer(s1, s2):
    # 获得交集
    len1 = len(list(set(s1).intersection(set(s2))))
    # 获得并集
    len2 = len(list(set(s1).union(set(s2))))
    return abs((len2 - len1))

# 字符串长度差异
def strLengthDiffer(str1, str2):
	return abs(len(str1) - len(str2))


# 字符集差距
def charSetDiffer(s1,s2):
    # 相同取0，同类取1，不同取10
    s = 0
    # 转成集合 2018.1.3改 start
    s1 = set(s1)
    s2 = set(s2)
    # 转成集合 2018.1.3改 end
    # 添加s1/s2区别主次逻辑 start
    # 主次判别因素的权重：是否含有特殊字符>长度较长>是否即有字符又有字母
    if ifContainSenstive(s1) and not ifContainSenstive(s2):
        pass
    elif ifContainSenstive(s1) and  ifContainSenstive(s2):
        if s1.__len__()<s2.__len__():
            temp = s2
            s2 = s1
            s1= temp
        elif s1.__len__()==s2.__len__():
            if not ifContainAlphaAndNum(s1):
                if ifContainAlphaAndNum(s2):
                    temp = s2
                    s2 = s1
                    s1 = temp
                else:
                    pass
        else:
            pass
    elif not ifContainSenstive(s1) and ifContainSenstive(s2):
        temp = s2
        s2 = s1
        s1 = temp
    else:
        # s1 s2都没有特殊字符时
        if s1.__len__()<s2.__len__():
            temp = s2
            s2 = s1
            s1= temp
        elif s1.__len__()==s2.__len__():
            if not ifContainAlphaAndNum(s1):
                if ifContainAlphaAndNum(s2):
                    temp = s2
                    s2 = s1
                    s1 = temp
                else:
                    pass
        else:
            pass
    # 添加s1/s2区别主次逻辑 end
    for x1 in s1:
        a = 10
        for x2 in s2:
            if (x1 == x2):
                a1 = 0
            else:
                # 数字字母间差异权值由10缩小到4
                if (x1.isalpha() == True):
                    if (x2.isalpha() == True):
                        a1 = 1
                    else:
                        a1 = 4
                elif (x1.isdigit() == True):
                    if (x2.isdigit() == True):
                        a1 = 1
                    else:
                        a1 = 4
                else:
                    if (x2.isdigit() == False and x2.isalpha() == False):
                        # 特殊符号间差异值为15
                        a1 = 10 #2018.1.3改 原版 a1 = 1
                    else:
                        # x2 是字符或者数字时 也就是说字符或者数字和特殊符号间的差异应该要大一些
                        a1 = 20
            if (a1 < a):
                a = a1
        s += a
    return s

# 最终得出的样本间的距离
def finaldistance(str1,str2):
    string_1 = str1
    string_2 = str2
    a = strLengthDiffer(string_1, string_2)
    b = charSetDiffer(string_1, string_2)
    s1 = n_grams(string_1)
    s2 = n_grams(string_2)
    c = groupdiffer(s1, s2)
    number = a+b+c
    return number

# 去除类的空值
def delespace(list_1):
    while '' in list_1:
        list_1.remove('')


# 样本与类间的距离
def calDisrance_sample_class(sample, class1):
    list_0 = []
    length = len(class1)
    for item in class1:
        list_0.append(finaldistance(sample, item))
    return sum(list_0)/length

# 类间距离
def internal_classes(class1, class2):
    list_0 = []
    m = len(class1)
    n = len(class2)
    for item1 in class1:
        for item2 in class2:
            list_0.append(finaldistance(item1, item2))
    return sum(list_0)/(m*n)

# 类内距离
def external_classes(class1):
    list_0 = []
    n = len(class1)
    for item1 in class1:
        for item2 in class1:
            list_0.append(finaldistance(item1, item2))
    return sum(list_0)/(n*n)




if __name__ == '__main__':
    # # 进行测试
    # conn = pymysql.connect(
    #     host='localhost',
    #     port=3306,
    #     user='root',
    #     passwd='123456',
    #     db='httpdata'
    # )
    # cur=conn.cursor()
    # sql="select modoA from unnormalparams"
    # cur.execute(sql)
    # rs = cur.fetchall()
    # string_0 = ''
    # list_0 = []
    # for x in rs:
    #     list_0.append(x[0])
    #
    #
    # conn.close()
    # cur.close()
    # # 输入样本
    # a = input()
    # # 样本与类间距离
    # distance1 = calDisrance_sample_class(a, list_0)
    # # 类内距离
    # distance2 = external_classes(list_0)

    print(charSetDiffer("##123","select * from table and '1'='1"))
    print(charSetDiffer("select * from table and '1'='1","##123"))
    print(charSetDiffer("a","123"))
    print(charSetDiffer("123","a"))
    print(charSetDiffer("ab","123"))
    print(charSetDiffer("123","ab"))
    print(charSetDiffer("ab1","123"))
    print(charSetDiffer("123","ab1"))
    print(charSetDiffer("absadcaaxcvaq","abhk-+"))
    print(charSetDiffer("abhk-+","absadcaaxcvaq"))
    print(charSetDiffer("abhk","absadcaaxcvaq"))
    print(charSetDiffer("absadcaaxcvaq","abhk"))
